# -*- coding:utf-8 -*-

import requests
from lxml import etree
from fake_useragent import UserAgent
from http import cookiejar
import json
from datetime import datetime
import time
import re
import csv
from pymongo import MongoClient


requests.packages.urllib3.disable_warnings()  # 忽略HTTPS安全警告

"""
中国化工制造网-企业名称、联系人、电话、地址
http://company.chemmade.com/list/c14-1.html
"""


class Test_Get():
    def __init__(self):
        #声明一个CookieJar对象实例来保存cookie
        # requests.utils.dict_from_cookiejar(html.cookies)  # 获取cookies转dict
        self.cookie = cookiejar.CookieJar()
        ua = UserAgent(use_cache_server=False)  # 禁用服务器缓存
        self.headers = {
            'User-Agent': ua.random,



        }


    def get_contents(self, cursor):
        postData = {
            'md5': '4426a43dcb6035113952fdc4814ca14f',
            'type': '2',
            'pageIndex': '3',
            'pageSize': '20',
            'isError': '0',
            '_': '1589335332556'

        }
        url = f"http://company.chemmade.com/list/c31-a20-{cursor}.html"
        html = HttpUtils.do_request("GET", url, self.headers, "")

        root = etree.HTML(html.text)
        for r in root.xpath('//div[@id="sjlist_content"]'):
            company_name = r.xpath('.//div[@class="sjlist_text"]/a/text()')[0]
            url_child = r.xpath('.//div[@class="sjlist_text"]/a/@href')[0]
            print(f"============{company_name}=============")
            html_child = HttpUtils.do_request("GET", url_child, self.headers, "")
            root_child = etree.HTML(html_child.text)
            info_root = root_child.xpath('//div[@class="lx_name"]/text()')
            if len(info_root) > 0:
                lxr = info_root[0]
                tel = "'" + info_root[2]
                tel1 = "'" + info_root[3]
                address = info_root[5]
                print(lxr)
                print(tel)
                print(tel1)
                print(address)
            else:
                info_root_1 = root_child.xpath('//div[@class="lx_name2"]/text()')
                lxr = info_root_1[1]
                tel = "'" + info_root_1[2]
                tel1 = "'" + info_root_1[3]
                address = info_root_1[5]
                if len(address) < 2:
                    address = info_root_1[6]
                print(lxr)
                print(tel)
                print(tel1)
                print(address)
            with open("化工企业.csv", "a+", encoding="utf-8-sig", newline="") as f:
                csv_writer = csv.writer(f)
                csv_writer.writerow(["其他", company_name, lxr, tel, tel1, address])


if __name__ == '__main__':
    test_get = Test_Get()
    for i in range(1, 11):
        print(f"============第【{i}】页=============")
        test_get.get_contents(i)
